package com.crawler.scheduler.remover;

import com.crawler.api.CrawlerRemover;
import com.crawler.model.CrawlerRequestInfo;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;

import java.util.Collections;
import java.util.HashSet;
import java.util.Set;

@Service
@Slf4j
public class DefaultCrawlerRemover implements CrawlerRemover {

    private volatile Set<String> visitedUrlSet = Collections.synchronizedSet(new HashSet<String>());

    @Override
    public boolean filter(CrawlerRequestInfo crawlerRequestInfo) {
        return !visitedUrlSet.add(crawlerRequestInfo.getUrl());
    }

    @Override
    public void clearRemover() {
        visitedUrlSet.clear();
    }

    @Override
    public int getUrlNum() {
        return visitedUrlSet.size();
    }
}
